{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Debugging and Testing Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code to Transform Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import zipfile\n",
    "url = 'data/kaggle-survey-2018.zip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['multipleChoiceResponses.csv', 'freeFormResponses.csv', 'SurveySchema.csv']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/matt/.env/pandas1/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3051: DtypeWarning: Columns (0,2,8,10,21,23,24,25,26,27,28,44,56,64,83,85,87,107,109,123,125,150,157,172,174,194,210,218,219,223,246,249,262,264,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,304,306,325,326,329,341,368,371,384,385,389,390,391,393,394) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "with zipfile.ZipFile(url) as z:\n",
    "    print(z.namelist())\n",
    "    kag = pd.read_csv(z.open('multipleChoiceResponses.csv'))\n",
    "    df = kag.iloc[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>23850</th>\n",
       "      <th>23851</th>\n",
       "      <th>23852</th>\n",
       "      <th>23853</th>\n",
       "      <th>23854</th>\n",
       "      <th>23855</th>\n",
       "      <th>23856</th>\n",
       "      <th>23857</th>\n",
       "      <th>23858</th>\n",
       "      <th>23859</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Time from Start to Finish (seconds)</th>\n",
       "      <td>710</td>\n",
       "      <td>434</td>\n",
       "      <td>718</td>\n",
       "      <td>621</td>\n",
       "      <td>731</td>\n",
       "      <td>1142</td>\n",
       "      <td>959</td>\n",
       "      <td>1758</td>\n",
       "      <td>641</td>\n",
       "      <td>751</td>\n",
       "      <td>...</td>\n",
       "      <td>820</td>\n",
       "      <td>683</td>\n",
       "      <td>57</td>\n",
       "      <td>122</td>\n",
       "      <td>348</td>\n",
       "      <td>575</td>\n",
       "      <td>131</td>\n",
       "      <td>370</td>\n",
       "      <td>36</td>\n",
       "      <td>502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q1</th>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>...</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q1_OTHER_TEXT</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q2</th>\n",
       "      <td>45-49</td>\n",
       "      <td>30-34</td>\n",
       "      <td>30-34</td>\n",
       "      <td>35-39</td>\n",
       "      <td>22-24</td>\n",
       "      <td>25-29</td>\n",
       "      <td>35-39</td>\n",
       "      <td>18-21</td>\n",
       "      <td>25-29</td>\n",
       "      <td>30-34</td>\n",
       "      <td>...</td>\n",
       "      <td>18-21</td>\n",
       "      <td>22-24</td>\n",
       "      <td>18-21</td>\n",
       "      <td>30-34</td>\n",
       "      <td>30-34</td>\n",
       "      <td>45-49</td>\n",
       "      <td>25-29</td>\n",
       "      <td>22-24</td>\n",
       "      <td>25-29</td>\n",
       "      <td>25-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q3</th>\n",
       "      <td>United States of America</td>\n",
       "      <td>Indonesia</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>India</td>\n",
       "      <td>Colombia</td>\n",
       "      <td>Chile</td>\n",
       "      <td>India</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>Hungary</td>\n",
       "      <td>...</td>\n",
       "      <td>India</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>France</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>Turkey</td>\n",
       "      <td>United Kingdom of Great Britain and Northern I...</td>\n",
       "      <td>Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q50_Part_5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Not enough incentives to share my work</td>\n",
       "      <td>Not enough incentives to share my work</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Not enough incentives to share my work</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q50_Part_6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I had never considered making my work easier f...</td>\n",
       "      <td>I had never considered making my work easier f...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q50_Part_7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q50_Part_8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q50_OTHER_TEXT</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>395 rows × 23859 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        1          2      \\\n",
       "Time from Start to Finish (seconds)                       710        434   \n",
       "Q1                                                     Female       Male   \n",
       "Q1_OTHER_TEXT                                              -1         -1   \n",
       "Q2                                                      45-49      30-34   \n",
       "Q3                                   United States of America  Indonesia   \n",
       "...                                                       ...        ...   \n",
       "Q50_Part_5                                                NaN        NaN   \n",
       "Q50_Part_6                                                NaN        NaN   \n",
       "Q50_Part_7                                                NaN        NaN   \n",
       "Q50_Part_8                                                NaN        NaN   \n",
       "Q50_OTHER_TEXT                                             -1         -1   \n",
       "\n",
       "                                                        3      \\\n",
       "Time from Start to Finish (seconds)                       718   \n",
       "Q1                                                     Female   \n",
       "Q1_OTHER_TEXT                                              -1   \n",
       "Q2                                                      30-34   \n",
       "Q3                                   United States of America   \n",
       "...                                                       ...   \n",
       "Q50_Part_5                                                NaN   \n",
       "Q50_Part_6                                                NaN   \n",
       "Q50_Part_7                                                NaN   \n",
       "Q50_Part_8                                                NaN   \n",
       "Q50_OTHER_TEXT                                             -1   \n",
       "\n",
       "                                                                      4      \\\n",
       "Time from Start to Finish (seconds)                                     621   \n",
       "Q1                                                                     Male   \n",
       "Q1_OTHER_TEXT                                                            -1   \n",
       "Q2                                                                    35-39   \n",
       "Q3                                                 United States of America   \n",
       "...                                                                     ...   \n",
       "Q50_Part_5                           Not enough incentives to share my work   \n",
       "Q50_Part_6                                                              NaN   \n",
       "Q50_Part_7                                                              NaN   \n",
       "Q50_Part_8                                                              NaN   \n",
       "Q50_OTHER_TEXT                                                           -1   \n",
       "\n",
       "                                                                      5      \\\n",
       "Time from Start to Finish (seconds)                                     731   \n",
       "Q1                                                                     Male   \n",
       "Q1_OTHER_TEXT                                                            -1   \n",
       "Q2                                                                    22-24   \n",
       "Q3                                                                    India   \n",
       "...                                                                     ...   \n",
       "Q50_Part_5                           Not enough incentives to share my work   \n",
       "Q50_Part_6                                                              NaN   \n",
       "Q50_Part_7                                                              NaN   \n",
       "Q50_Part_8                                                              NaN   \n",
       "Q50_OTHER_TEXT                                                           -1   \n",
       "\n",
       "                                                                                 6      \\\n",
       "Time from Start to Finish (seconds)                                               1142   \n",
       "Q1                                                                                Male   \n",
       "Q1_OTHER_TEXT                                                                       -1   \n",
       "Q2                                                                               25-29   \n",
       "Q3                                                                            Colombia   \n",
       "...                                                                                ...   \n",
       "Q50_Part_5                                                                         NaN   \n",
       "Q50_Part_6                           I had never considered making my work easier f...   \n",
       "Q50_Part_7                                                                         NaN   \n",
       "Q50_Part_8                                                                         NaN   \n",
       "Q50_OTHER_TEXT                                                                      -1   \n",
       "\n",
       "                                                                                 7      \\\n",
       "Time from Start to Finish (seconds)                                                959   \n",
       "Q1                                                                                Male   \n",
       "Q1_OTHER_TEXT                                                                       -1   \n",
       "Q2                                                                               35-39   \n",
       "Q3                                                                               Chile   \n",
       "...                                                                                ...   \n",
       "Q50_Part_5                                                                         NaN   \n",
       "Q50_Part_6                           I had never considered making my work easier f...   \n",
       "Q50_Part_7                                                                         NaN   \n",
       "Q50_Part_8                                                                         NaN   \n",
       "Q50_OTHER_TEXT                                                                      -1   \n",
       "\n",
       "                                                                      8      \\\n",
       "Time from Start to Finish (seconds)                                    1758   \n",
       "Q1                                                                     Male   \n",
       "Q1_OTHER_TEXT                                                            -1   \n",
       "Q2                                                                    18-21   \n",
       "Q3                                                                    India   \n",
       "...                                                                     ...   \n",
       "Q50_Part_5                           Not enough incentives to share my work   \n",
       "Q50_Part_6                                                              NaN   \n",
       "Q50_Part_7                                                              NaN   \n",
       "Q50_Part_8                                                              NaN   \n",
       "Q50_OTHER_TEXT                                                           -1   \n",
       "\n",
       "                                      9        10     ...   23850   23851  \\\n",
       "Time from Start to Finish (seconds)     641      751  ...     820     683   \n",
       "Q1                                     Male     Male  ...  Female    Male   \n",
       "Q1_OTHER_TEXT                            -1       -1  ...      -1      -1   \n",
       "Q2                                    25-29    30-34  ...   18-21   22-24   \n",
       "Q3                                   Turkey  Hungary  ...   India  Turkey   \n",
       "...                                     ...      ...  ...     ...     ...   \n",
       "Q50_Part_5                              NaN      NaN  ...     NaN     NaN   \n",
       "Q50_Part_6                              NaN      NaN  ...     NaN     NaN   \n",
       "Q50_Part_7                              NaN      NaN  ...     NaN     NaN   \n",
       "Q50_Part_8                              NaN      NaN  ...     NaN     NaN   \n",
       "Q50_OTHER_TEXT                           -1       -1  ...      -1      -1   \n",
       "\n",
       "                                      23852   23853   23854   23855   23856  \\\n",
       "Time from Start to Finish (seconds)      57     122     348     575     131   \n",
       "Q1                                   Female  Female    Male    Male  Female   \n",
       "Q1_OTHER_TEXT                            -1      -1      -1      -1      -1   \n",
       "Q2                                    18-21   30-34   30-34   45-49   25-29   \n",
       "Q3                                   Turkey  Turkey  Turkey  France  Turkey   \n",
       "...                                     ...     ...     ...     ...     ...   \n",
       "Q50_Part_5                              NaN     NaN     NaN     NaN     NaN   \n",
       "Q50_Part_6                              NaN     NaN     NaN     NaN     NaN   \n",
       "Q50_Part_7                              NaN     NaN     NaN     NaN     NaN   \n",
       "Q50_Part_8                              NaN     NaN     NaN     NaN     NaN   \n",
       "Q50_OTHER_TEXT                           -1      -1      -1      -1      -1   \n",
       "\n",
       "                                      23857  \\\n",
       "Time from Start to Finish (seconds)     370   \n",
       "Q1                                     Male   \n",
       "Q1_OTHER_TEXT                            -1   \n",
       "Q2                                    22-24   \n",
       "Q3                                   Turkey   \n",
       "...                                     ...   \n",
       "Q50_Part_5                              NaN   \n",
       "Q50_Part_6                              NaN   \n",
       "Q50_Part_7                              NaN   \n",
       "Q50_Part_8                              NaN   \n",
       "Q50_OTHER_TEXT                           -1   \n",
       "\n",
       "                                                                                 23858  \\\n",
       "Time from Start to Finish (seconds)                                                 36   \n",
       "Q1                                                                                Male   \n",
       "Q1_OTHER_TEXT                                                                       -1   \n",
       "Q2                                                                               25-29   \n",
       "Q3                                   United Kingdom of Great Britain and Northern I...   \n",
       "...                                                                                ...   \n",
       "Q50_Part_5                                                                         NaN   \n",
       "Q50_Part_6                                                                         NaN   \n",
       "Q50_Part_7                                                                         NaN   \n",
       "Q50_Part_8                                                                         NaN   \n",
       "Q50_OTHER_TEXT                                                                      -1   \n",
       "\n",
       "                                     23859  \n",
       "Time from Start to Finish (seconds)    502  \n",
       "Q1                                    Male  \n",
       "Q1_OTHER_TEXT                           -1  \n",
       "Q2                                   25-29  \n",
       "Q3                                   Spain  \n",
       "...                                    ...  \n",
       "Q50_Part_5                             NaN  \n",
       "Q50_Part_6                             NaN  \n",
       "Q50_Part_7                             NaN  \n",
       "Q50_Part_8                             NaN  \n",
       "Q50_OTHER_TEXT                          -1  \n",
       "\n",
       "[395 rows x 23859 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Time from Start to Finish (seconds)    object\n",
       "Q1                                     object\n",
       "Q1_OTHER_TEXT                          object\n",
       "Q2                                     object\n",
       "Q3                                     object\n",
       "                                        ...  \n",
       "Q50_Part_5                             object\n",
       "Q50_Part_6                             object\n",
       "Q50_Part_7                             object\n",
       "Q50_Part_8                             object\n",
       "Q50_OTHER_TEXT                         object\n",
       "Length: 395, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Male                       19430\n",
       "Female                      4010\n",
       "Prefer not to say            340\n",
       "Prefer to self-describe       79\n",
       "Name: Q1, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Q1.value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def tweak_kag(df):\n",
    "    na_mask = df.Q9.isna()\n",
    "    hide_mask = df.Q9.str.startswith('I do not').fillna(False)\n",
    "    df = df[~na_mask & ~hide_mask]\n",
    "\n",
    "\n",
    "    q1 = (df.Q1\n",
    "      .replace({'Prefer not to say': 'Another',\n",
    "               'Prefer to self-describe': 'Another'})\n",
    "      .rename('Gender')\n",
    "    )\n",
    "    q2 = df.Q2.str.slice(0,2).astype(int).rename('Age')\n",
    "    def limit_countries(val):\n",
    "        if val in  {'United States of America', 'India', 'China'}:\n",
    "            return val\n",
    "        return 'Another'\n",
    "    q3 = df.Q3.apply(limit_countries).rename('Country')\n",
    "\n",
    "\n",
    "    q4 = (df.Q4\n",
    "     .replace({'Master’s degree': 18,\n",
    "     'Bachelor’s degree': 16,\n",
    "     'Doctoral degree': 20,\n",
    "     'Some college/university study without earning a bachelor’s degree': 13,\n",
    "     'Professional degree': 19,\n",
    "     'I prefer not to answer': None,\n",
    "     'No formal education past high school': 12})\n",
    "     .fillna(11)\n",
    "     .rename('Edu')\n",
    "    )\n",
    "\n",
    "\n",
    "    def only_cs_stat_val(val):\n",
    "        if val not in {'cs', 'eng', 'stat'}:\n",
    "            return 'another'\n",
    "        return val\n",
    "\n",
    "\n",
    "    q5 = (df.Q5\n",
    "            .replace({\n",
    "                'Computer science (software engineering, etc.)': 'cs',\n",
    "                'Engineering (non-computer focused)': 'eng',\n",
    "                'Mathematics or statistics': 'stat'})\n",
    "             .apply(only_cs_stat_val)\n",
    "             .rename('Studies'))\n",
    "    def limit_occupation(val):\n",
    "        if val in {'Student', 'Data Scientist', 'Software Engineer', 'Not employed',\n",
    "                  'Data Engineer'}:\n",
    "            return val\n",
    "        return 'Another'\n",
    "\n",
    "\n",
    "    q6 = df.Q6.apply(limit_occupation).rename('Occupation')\n",
    "\n",
    "\n",
    "    q8 = (df.Q8\n",
    "      .str.replace('+', '')\n",
    "      .str.split('-', expand=True)\n",
    "      .iloc[:,0]\n",
    "      .fillna(-1)\n",
    "      .astype(int)\n",
    "      .rename('Experience')\n",
    "    )\n",
    "\n",
    "\n",
    "    q9 = (df.Q9\n",
    "     .str.replace('+','')\n",
    "     .str.replace(',','')\n",
    "     .str.replace('500000', '500')\n",
    "     .str.replace('I do not wish to disclose my approximate yearly compensation','')\n",
    "     .str.split('-', expand=True)\n",
    "     .iloc[:,0]\n",
    "     .astype(int)\n",
    "     .mul(1000)\n",
    "     .rename('Salary'))\n",
    "    return pd.concat([q1, q2, q3, q4, q5, q6, q8, q9], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Country</th>\n",
       "      <th>Edu</th>\n",
       "      <th>Studies</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Experience</th>\n",
       "      <th>Salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>30</td>\n",
       "      <td>Another</td>\n",
       "      <td>16.0</td>\n",
       "      <td>eng</td>\n",
       "      <td>Another</td>\n",
       "      <td>5</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Female</td>\n",
       "      <td>30</td>\n",
       "      <td>United States of America</td>\n",
       "      <td>18.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Male</td>\n",
       "      <td>22</td>\n",
       "      <td>India</td>\n",
       "      <td>18.0</td>\n",
       "      <td>stat</td>\n",
       "      <td>Another</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Male</td>\n",
       "      <td>35</td>\n",
       "      <td>Another</td>\n",
       "      <td>20.0</td>\n",
       "      <td>another</td>\n",
       "      <td>Another</td>\n",
       "      <td>10</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Male</td>\n",
       "      <td>18</td>\n",
       "      <td>India</td>\n",
       "      <td>18.0</td>\n",
       "      <td>another</td>\n",
       "      <td>Another</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23844</th>\n",
       "      <td>Male</td>\n",
       "      <td>30</td>\n",
       "      <td>Another</td>\n",
       "      <td>18.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>Software Engineer</td>\n",
       "      <td>10</td>\n",
       "      <td>90000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23845</th>\n",
       "      <td>Male</td>\n",
       "      <td>22</td>\n",
       "      <td>Another</td>\n",
       "      <td>18.0</td>\n",
       "      <td>stat</td>\n",
       "      <td>Student</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23854</th>\n",
       "      <td>Male</td>\n",
       "      <td>30</td>\n",
       "      <td>Another</td>\n",
       "      <td>20.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>Another</td>\n",
       "      <td>5</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23855</th>\n",
       "      <td>Male</td>\n",
       "      <td>45</td>\n",
       "      <td>Another</td>\n",
       "      <td>20.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>Another</td>\n",
       "      <td>5</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23857</th>\n",
       "      <td>Male</td>\n",
       "      <td>22</td>\n",
       "      <td>Another</td>\n",
       "      <td>18.0</td>\n",
       "      <td>cs</td>\n",
       "      <td>Software Engineer</td>\n",
       "      <td>0</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15429 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Gender  Age                   Country   Edu  Studies  \\\n",
       "2        Male   30                   Another  16.0      eng   \n",
       "3      Female   30  United States of America  18.0       cs   \n",
       "5        Male   22                     India  18.0     stat   \n",
       "7        Male   35                   Another  20.0  another   \n",
       "8        Male   18                     India  18.0  another   \n",
       "...       ...  ...                       ...   ...      ...   \n",
       "23844    Male   30                   Another  18.0       cs   \n",
       "23845    Male   22                   Another  18.0     stat   \n",
       "23854    Male   30                   Another  20.0       cs   \n",
       "23855    Male   45                   Another  20.0       cs   \n",
       "23857    Male   22                   Another  18.0       cs   \n",
       "\n",
       "              Occupation  Experience  Salary  \n",
       "2                Another           5   10000  \n",
       "3         Data Scientist           0       0  \n",
       "5                Another           0       0  \n",
       "7                Another          10   10000  \n",
       "8                Another           0       0  \n",
       "...                  ...         ...     ...  \n",
       "23844  Software Engineer          10   90000  \n",
       "23845            Student           0       0  \n",
       "23854            Another           5   10000  \n",
       "23855            Another           5  250000  \n",
       "23857  Software Engineer           0   10000  \n",
       "\n",
       "[15429 rows x 8 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweak_kag(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Gender         object\n",
       "Age             int64\n",
       "Country        object\n",
       "Edu           float64\n",
       "Studies        object\n",
       "Occupation     object\n",
       "Experience      int64\n",
       "Salary          int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweak_kag(df).dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Country\n",
       "Another                     0.289827\n",
       "China                       0.252974\n",
       "India                       0.167335\n",
       "United States of America    0.354125\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag = tweak_kag(df)\n",
    "(kag\n",
    "    .groupby('Country')\n",
    "    .apply(lambda g: g.Salary.corr(g.Experience))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply Performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def limit_countries(val):\n",
    "     if val in  {'United States of America', 'India', 'China'}:\n",
    "         return val\n",
    "     return 'Another'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.35 ms ± 1.71 ms per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "q3 = df.Q3.apply(limit_countries).rename('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30.8 ms ± 6.26 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "other_values = df.Q3.value_counts().iloc[3:].index\n",
    "q3_2 = df.Q3.replace(other_values, 'Another')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.53 ms ± 119 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "values = {'United States of America', 'India', 'China'}\n",
    "q3_3 = df.Q3.where(df.Q3.isin(values), 'Another')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.32 ms ± 1.15 ms per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "values = {'United States of America', 'India', 'China'}\n",
    "q3_4 = pd.Series(np.where(df.Q3.isin(values), df.Q3, 'Another'), \n",
    "     index=df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'q3' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-62ef4889ac47>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mq3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mequals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq3_2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'q3' is not defined"
     ]
    }
   ],
   "source": [
    "q3.equals(q3_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "q3.equals(q3_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q3.equals(q3_4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There's more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def limit_countries(val):\n",
    "     if val in  {'United States of America', 'India', 'China'}:\n",
    "         return val\n",
    "     return 'Another'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "q3 = df.Q3.apply(limit_countries).rename('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def debug(something):\n",
    "    # what is something? A cell, series, dataframe?\n",
    "    print(type(something), something)\n",
    "    1/0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'str'> United States of America\n"
     ]
    },
    {
     "ename": "ZeroDivisionError",
     "evalue": "division by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-d44840fcaee6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mq3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.env/pandas1/lib/python3.7/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   3846\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3847\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3848\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3850\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-21-8d0588b1c522>\u001b[0m in \u001b[0;36mdebug\u001b[0;34m(something)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;31m# what is something? A cell, series, dataframe?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msomething\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msomething\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
     ]
    }
   ],
   "source": [
    "q3.apply(debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "the_item = None\n",
    "def debug(something):\n",
    "    global the_item\n",
    "    the_item = something\n",
    "    return something"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "_ = q3.apply(debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Another'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "the_item"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Improving Apply Performance with Dask, Pandarell, Swifter, and More"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use standard multiprocessing data transfer (pipe) to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "from pandarallel import pandarallel\n",
    "pandarallel.initialize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def limit_countries(val):\n",
    "     if val in  {'United States of America', 'India', 'China'}:\n",
    "         return val\n",
    "     return 'Another'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "117 ms ± 11.5 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "res_p = df.Q3.parallel_apply(limit_countries).rename('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import swifter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/matt/.env/pandas1/lib/python3.7/site-packages/tqdm/std.py:658: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9707a7a972284329996376005c0fa9a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a534cfa7c61414c916a110a35b364bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a9b9a4446a348bb8a064f455522e04c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27c0e745bcc74126adcaefe6b38b5914",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed2cf1a4e86647d5ab478d973c72c5f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9ab7802f66c4b06bb00e844f56ae180",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd9ef7225820431db3fbb027e93d8f0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb1f2667bb424522988365393acd2371",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5017212f5ace4698b4d24d260420e7b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1ee29ee2a384d8ea0d1ed3ba2345b46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de2750a7c32841ba9ba88e3025679530",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48e4478d489c4442bec7ccdb4baec968",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf588f1dfeae42e39664f5591b5a1f3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94427a7e917b4c879926599b491f9d39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8edb2248549d40319294380ba59959d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a87c85ef85b543e9af39421494195c3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae8c2a3905f04fbdb798bb2b79f209e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5f0b9cd6c6f470abce0d567c4bfd97d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a871d831ec14103a20b8b582e367dcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e646e18f818c4675a29b8a4d8b44e76c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aece50464eba48c58b4640d306fd313f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e915a1c71c5943b1b6fa3af10b506a67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3a28a97fc004a3ca85a81a419e34c74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efd2bfcc950644768938fe468a70009d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1e304a1a4ae4b48ad761c30058fb76c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cce9be2bb7b24ab382b743998e2ce117",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9979ff13e464435a86fb92ca2f63fc26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "246a4adbc8444b0d801d82d3d088f59f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e773aba0f2d84fe6be7e05dbdde75c68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ca5a4763a864248942473db717166bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bafd730d295042588c9e1f3f4e08b674",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf1925801fe544cdb29f43acea95af88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c804f573122466887d30ac84002208a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f8bac46a1d34c869c5e9167ec5ebb30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9e43891fa564304b63ec136535598c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cae0c1593424435b9374571143f5e5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39900aef24a84278b147511a035fde17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0783b63e9482425bbf169783e544e66f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "109f208df7f744e3aeb6e087ecf631fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "050f8f5d571b4721aaf9898ae8b9e5b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8804ee93eeb410489c8a04a0062ba94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00abe2438c14424297212ea6f68344a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b150bef5e0a44f34aab668fdc3bed124",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cbcb57f1573420ea1449c349a491d10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8954386e7c54f1d859fa9ae751d0a1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9dc648ccdbc546ee856fd981749b851b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5738f96d2d2f49a69b5aaf2f84c78c3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d551eb5dcdd44d0b8a4e0ebb45895027",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d35705cdfa44b6bba3176ca2dc865cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e99eda6c9e14d5ba949b850f5784c89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47c5ee7a83994cc99a9f56e73cfb72dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e920126d4944b7785378cd02f010d46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5020ef8fd9db4dbf95d484c227500af7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f5db94d7c024ab3a1a4ef97286c78ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bcee8277c784481a7f6115b6927404f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e4ae71319b84df2878f088a20f0961e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cbe431cb18446a79698bed9915492e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51cf39e83b344842ae0980dc9ef6f968",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "702cb1f74abf40c38aa32cd247809446",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ab2171e08e340ff934a29be8675b800",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a44f7d1a0081498687479dafab98f59e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab3cb1a51a8f43ae94522463e4f58559",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6264d84b1a84d21a9084cecbdd3913e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55fe027d346040b1b71a017ae9d19f64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a345537134764aba985c40baebef4672",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ef462f9d21147ea84473c31a5319e78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6817335cfe04f2dbac4c929568a762a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92b2c84ae5874b2e9df2cf4de341d884",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3015b31f21b4f8998a1c75c70cf7348",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0edb9042dc1a488786524ca9fbb72ba6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f7e01f2a69b43cd9137ae4f54524726",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf20bf297fb14a7da80d4b019b8f20da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08bc4e9d6b5b4983a6aa6009bab67fee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01cbf6306b98444698dc0449e4f8ba52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2189d1eab2d408f9268713b1ff1fdec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbbe4c495643444985123316c43d7871",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63a9b4bacd93452d90dd455e6bdc9800",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f41aeca48baa4b9ca9000f7483c0d373",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cad4722e8bb14cb198e98ed96b619915",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "706150bfcd6d4b5f968ed830f2e3a794",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80e5863b7bc34bccac244c20ac313637",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=23859.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "179 ms ± 81.1 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "res_s = df.Q3.swifter.apply(limit_countries).rename('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "710 ms ± 72.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "res_d = (dask.dataframe.from_pandas(\n",
    "       df, npartitions=4)\n",
    "   .map_partitions(lambda df: df.Q3.apply(limit_countries))\n",
    "   .rename('Countries')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "np_fn = np.vectorize(limit_countries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "414 ms ± 16.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "res_v = df.Q3.apply(np_fn).rename('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numba import jit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "@jit\n",
    "def limit_countries2(val):\n",
    "     if val in  ['United States of America', 'India', 'China']:\n",
    "         return val\n",
    "     return 'Another'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106 ms ± 45.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "res_n = df.Q3.apply(limit_countries2).rename('Country')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspecting Code "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import zipfile\n",
    "url = 'data/kaggle-survey-2018.zip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/matt/.env/pandas1/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3051: DtypeWarning: Columns (0,2,8,10,21,23,24,25,26,27,28,44,56,64,83,85,87,107,109,123,125,150,157,172,174,194,210,218,219,223,246,249,262,264,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,304,306,325,326,329,341,368,371,384,385,389,390,391,393,394) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "with zipfile.ZipFile(url) as z:\n",
    "    kag = pd.read_csv(z.open('multipleChoiceResponses.csv'))\n",
    "    df = kag.iloc[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "comment_questions": false,
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "df.Q3.apply?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "comment_questions": false,
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "df.Q3.apply??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'pandas._libs.lib' from '/Users/matt/.env/pandas1/lib/python3.7/site-packages/pandas/_libs/lib.cpython-37m-darwin.so'>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas.core.series\n",
    "pandas.core.series.lib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "comment_questions": false,
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "pandas.core.series.lib.map_infer??"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There's more..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Debugging in Jupyter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import zipfile\n",
    "url = 'data/kaggle-survey-2018.zip'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "with zipfile.ZipFile(url) as z:\n",
    "    kag = pd.read_csv(z.open('multipleChoiceResponses.csv'))\n",
    "    df = kag.iloc[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add1(x):\n",
    "    return x + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can only concatenate str (not \"int\") to str",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-61-c52fc69777f1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mQ3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0madd1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.env/pandas1/lib/python3.7/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   3846\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3847\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3848\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3850\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-60-f4e0c9584bd4>\u001b[0m in \u001b[0;36madd1\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0madd1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: can only concatenate str (not \"int\") to str"
     ]
    }
   ],
   "source": [
    "df.Q3.apply(add1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.debugger import set_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add1(x):\n",
    "    set_trace()\n",
    "    return x + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "df.Q3.apply(add1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There's more..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Managing data integrity with Great Expectations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "kag = tweak_kag(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "import great_expectations as ge\n",
    "kag_ge = ge.from_pandas(kag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['autoinspect',\n",
       " 'batch_id',\n",
       " 'batch_kwargs',\n",
       " 'batch_markers',\n",
       " 'batch_parameters',\n",
       " 'column_aggregate_expectation',\n",
       " 'column_map_expectation',\n",
       " 'column_pair_map_expectation',\n",
       " 'discard_failing_expectations',\n",
       " 'edit_expectation_suite',\n",
       " 'expect_column_bootstrapped_ks_test_p_value_to_be_greater_than',\n",
       " 'expect_column_chisquare_test_p_value_to_be_greater_than',\n",
       " 'expect_column_distinct_values_to_be_in_set',\n",
       " 'expect_column_distinct_values_to_contain_set',\n",
       " 'expect_column_distinct_values_to_equal_set',\n",
       " 'expect_column_kl_divergence_to_be_less_than',\n",
       " 'expect_column_max_to_be_between',\n",
       " 'expect_column_mean_to_be_between',\n",
       " 'expect_column_median_to_be_between',\n",
       " 'expect_column_min_to_be_between',\n",
       " 'expect_column_most_common_value_to_be_in_set',\n",
       " 'expect_column_pair_values_A_to_be_greater_than_B',\n",
       " 'expect_column_pair_values_to_be_equal',\n",
       " 'expect_column_pair_values_to_be_in_set',\n",
       " 'expect_column_parameterized_distribution_ks_test_p_value_to_be_greater_than',\n",
       " 'expect_column_proportion_of_unique_values_to_be_between',\n",
       " 'expect_column_quantile_values_to_be_between',\n",
       " 'expect_column_stdev_to_be_between',\n",
       " 'expect_column_sum_to_be_between',\n",
       " 'expect_column_to_exist',\n",
       " 'expect_column_unique_value_count_to_be_between',\n",
       " 'expect_column_value_lengths_to_be_between',\n",
       " 'expect_column_value_lengths_to_equal',\n",
       " 'expect_column_values_to_be_between',\n",
       " 'expect_column_values_to_be_dateutil_parseable',\n",
       " 'expect_column_values_to_be_decreasing',\n",
       " 'expect_column_values_to_be_in_set',\n",
       " 'expect_column_values_to_be_in_type_list',\n",
       " 'expect_column_values_to_be_increasing',\n",
       " 'expect_column_values_to_be_json_parseable',\n",
       " 'expect_column_values_to_be_null',\n",
       " 'expect_column_values_to_be_of_type',\n",
       " 'expect_column_values_to_be_unique',\n",
       " 'expect_column_values_to_match_json_schema',\n",
       " 'expect_column_values_to_match_regex',\n",
       " 'expect_column_values_to_match_regex_list',\n",
       " 'expect_column_values_to_match_strftime_format',\n",
       " 'expect_column_values_to_not_be_in_set',\n",
       " 'expect_column_values_to_not_be_null',\n",
       " 'expect_column_values_to_not_match_regex',\n",
       " 'expect_column_values_to_not_match_regex_list',\n",
       " 'expect_multicolumn_values_to_be_unique',\n",
       " 'expect_table_column_count_to_be_between',\n",
       " 'expect_table_column_count_to_equal',\n",
       " 'expect_table_columns_to_match_ordered_list',\n",
       " 'expect_table_row_count_to_be_between',\n",
       " 'expect_table_row_count_to_equal',\n",
       " 'expectation',\n",
       " 'expectation_suite_name',\n",
       " 'find_expectation_indexes',\n",
       " 'find_expectations',\n",
       " 'from_dataset',\n",
       " 'get_column_count',\n",
       " 'get_column_count_in_range',\n",
       " 'get_column_hist',\n",
       " 'get_column_max',\n",
       " 'get_column_mean',\n",
       " 'get_column_median',\n",
       " 'get_column_min',\n",
       " 'get_column_modes',\n",
       " 'get_column_nonnull_count',\n",
       " 'get_column_partition',\n",
       " 'get_column_quantiles',\n",
       " 'get_column_stdev',\n",
       " 'get_column_sum',\n",
       " 'get_column_unique_count',\n",
       " 'get_column_value_counts',\n",
       " 'get_config_value',\n",
       " 'get_default_expectation_arguments',\n",
       " 'get_evaluation_parameter',\n",
       " 'get_expectation_suite',\n",
       " 'get_expectations_config',\n",
       " 'get_row_count',\n",
       " 'get_table_columns',\n",
       " 'hashable_getters',\n",
       " 'multicolumn_map_expectation',\n",
       " 'profile',\n",
       " 'remove_expectation',\n",
       " 'save_expectation_suite',\n",
       " 'set_config_value',\n",
       " 'set_default_expectation_argument',\n",
       " 'set_evaluation_parameter',\n",
       " 'test_column_aggregate_expectation_function',\n",
       " 'test_column_map_expectation_function',\n",
       " 'test_expectation_function',\n",
       " 'validate']"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted([x for x in set(dir(kag_ge)) - set(dir(kag))\n",
    "    if not x.startswith('_')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  \"result\": {},\n",
       "  \"meta\": {},\n",
       "  \"expectation_config\": {\n",
       "    \"meta\": {},\n",
       "    \"expectation_type\": \"expect_column_to_exist\",\n",
       "    \"kwargs\": {\n",
       "      \"column\": \"Salary\",\n",
       "      \"result_format\": \"BASIC\"\n",
       "    }\n",
       "  },\n",
       "  \"exception_info\": null,\n",
       "  \"success\": true\n",
       "}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag_ge.expect_column_to_exist('Salary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  \"result\": {\n",
       "    \"observed_value\": 43869.66102793441,\n",
       "    \"element_count\": 15429,\n",
       "    \"missing_count\": 0,\n",
       "    \"missing_percent\": 0.0\n",
       "  },\n",
       "  \"meta\": {},\n",
       "  \"expectation_config\": {\n",
       "    \"meta\": {},\n",
       "    \"expectation_type\": \"expect_column_mean_to_be_between\",\n",
       "    \"kwargs\": {\n",
       "      \"column\": \"Salary\",\n",
       "      \"min_value\": 10000,\n",
       "      \"max_value\": 100000,\n",
       "      \"result_format\": \"BASIC\"\n",
       "    }\n",
       "  },\n",
       "  \"exception_info\": null,\n",
       "  \"success\": true\n",
       "}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag_ge.expect_column_mean_to_be_between(\n",
    "   'Salary', min_value=10_000, max_value=100_000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  \"result\": {\n",
       "    \"element_count\": 15429,\n",
       "    \"missing_count\": 0,\n",
       "    \"missing_percent\": 0.0,\n",
       "    \"unexpected_count\": 0,\n",
       "    \"unexpected_percent\": 0.0,\n",
       "    \"unexpected_percent_nonmissing\": 0.0,\n",
       "    \"partial_unexpected_list\": []\n",
       "  },\n",
       "  \"meta\": {},\n",
       "  \"expectation_config\": {\n",
       "    \"meta\": {},\n",
       "    \"expectation_type\": \"expect_column_values_to_be_between\",\n",
       "    \"kwargs\": {\n",
       "      \"column\": \"Salary\",\n",
       "      \"min_value\": 0,\n",
       "      \"max_value\": 500000,\n",
       "      \"result_format\": \"BASIC\"\n",
       "    }\n",
       "  },\n",
       "  \"exception_info\": null,\n",
       "  \"success\": true\n",
       "}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag_ge.expect_column_values_to_be_between(\n",
    "   'Salary', min_value=0, max_value=500_000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  \"result\": {\n",
       "    \"element_count\": 15429,\n",
       "    \"unexpected_count\": 0,\n",
       "    \"unexpected_percent\": 0.0,\n",
       "    \"partial_unexpected_list\": []\n",
       "  },\n",
       "  \"meta\": {},\n",
       "  \"expectation_config\": {\n",
       "    \"meta\": {},\n",
       "    \"expectation_type\": \"expect_column_values_to_not_be_null\",\n",
       "    \"kwargs\": {\n",
       "      \"column\": \"Salary\",\n",
       "      \"result_format\": \"BASIC\"\n",
       "    }\n",
       "  },\n",
       "  \"exception_info\": null,\n",
       "  \"success\": true\n",
       "}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag_ge.expect_column_values_to_not_be_null('Salary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  \"result\": {\n",
       "    \"element_count\": 15429,\n",
       "    \"missing_count\": 0,\n",
       "    \"missing_percent\": 0.0,\n",
       "    \"unexpected_count\": 0,\n",
       "    \"unexpected_percent\": 0.0,\n",
       "    \"unexpected_percent_nonmissing\": 0.0,\n",
       "    \"partial_unexpected_list\": []\n",
       "  },\n",
       "  \"meta\": {},\n",
       "  \"expectation_config\": {\n",
       "    \"meta\": {},\n",
       "    \"expectation_type\": \"expect_column_values_to_match_regex\",\n",
       "    \"kwargs\": {\n",
       "      \"column\": \"Country\",\n",
       "      \"regex\": \"America|India|Another|China\",\n",
       "      \"result_format\": \"BASIC\"\n",
       "    }\n",
       "  },\n",
       "  \"exception_info\": null,\n",
       "  \"success\": true\n",
       "}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag_ge.expect_column_values_to_match_regex(\n",
    "    'Country', r'America|India|Another|China')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  \"result\": {\n",
       "    \"observed_value\": \"int64\"\n",
       "  },\n",
       "  \"meta\": {},\n",
       "  \"expectation_config\": {\n",
       "    \"meta\": {},\n",
       "    \"expectation_type\": \"_expect_column_values_to_be_of_type__aggregate\",\n",
       "    \"kwargs\": {\n",
       "      \"column\": \"Salary\",\n",
       "      \"type_\": \"int\",\n",
       "      \"result_format\": \"BASIC\"\n",
       "    }\n",
       "  },\n",
       "  \"exception_info\": null,\n",
       "  \"success\": true\n",
       "}"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag_ge.expect_column_values_to_be_of_type(\n",
    "   'Salary', type_='int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "kag_ge.save_expectation_suite('kaggle_expectations.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  \"evaluation_parameters\": {},\n",
       "  \"meta\": {\n",
       "    \"great_expectations.__version__\": \"0.9.2\",\n",
       "    \"expectation_suite_name\": \"default\",\n",
       "    \"run_id\": \"20200224T221709.246886Z\",\n",
       "    \"batch_kwargs\": {\n",
       "      \"ge_batch_id\": \"65ebc1d8-5753-11ea-a940-a45e60ecc33f\"\n",
       "    },\n",
       "    \"batch_markers\": {},\n",
       "    \"batch_parameters\": {}\n",
       "  },\n",
       "  \"statistics\": {\n",
       "    \"evaluated_expectations\": 6,\n",
       "    \"successful_expectations\": 6,\n",
       "    \"unsuccessful_expectations\": 0,\n",
       "    \"success_percent\": 100.0\n",
       "  },\n",
       "  \"results\": [\n",
       "    {\n",
       "      \"result\": {},\n",
       "      \"meta\": {},\n",
       "      \"expectation_config\": {\n",
       "        \"meta\": {},\n",
       "        \"expectation_type\": \"expect_column_to_exist\",\n",
       "        \"kwargs\": {\n",
       "          \"column\": \"Salary\"\n",
       "        }\n",
       "      },\n",
       "      \"exception_info\": {\n",
       "        \"raised_exception\": false,\n",
       "        \"exception_message\": null,\n",
       "        \"exception_traceback\": null\n",
       "      },\n",
       "      \"success\": true\n",
       "    },\n",
       "    {\n",
       "      \"result\": {\n",
       "        \"observed_value\": 43869.66102793441,\n",
       "        \"element_count\": 15429,\n",
       "        \"missing_count\": 0,\n",
       "        \"missing_percent\": 0.0\n",
       "      },\n",
       "      \"meta\": {},\n",
       "      \"expectation_config\": {\n",
       "        \"meta\": {},\n",
       "        \"expectation_type\": \"expect_column_mean_to_be_between\",\n",
       "        \"kwargs\": {\n",
       "          \"column\": \"Salary\",\n",
       "          \"min_value\": 10000,\n",
       "          \"max_value\": 100000\n",
       "        }\n",
       "      },\n",
       "      \"exception_info\": {\n",
       "        \"raised_exception\": false,\n",
       "        \"exception_message\": null,\n",
       "        \"exception_traceback\": null\n",
       "      },\n",
       "      \"success\": true\n",
       "    },\n",
       "    {\n",
       "      \"result\": {\n",
       "        \"element_count\": 15429,\n",
       "        \"missing_count\": 0,\n",
       "        \"missing_percent\": 0.0,\n",
       "        \"unexpected_count\": 0,\n",
       "        \"unexpected_percent\": 0.0,\n",
       "        \"unexpected_percent_nonmissing\": 0.0,\n",
       "        \"partial_unexpected_list\": []\n",
       "      },\n",
       "      \"meta\": {},\n",
       "      \"expectation_config\": {\n",
       "        \"meta\": {},\n",
       "        \"expectation_type\": \"expect_column_values_to_be_between\",\n",
       "        \"kwargs\": {\n",
       "          \"column\": \"Salary\",\n",
       "          \"min_value\": 0,\n",
       "          \"max_value\": 500000\n",
       "        }\n",
       "      },\n",
       "      \"exception_info\": {\n",
       "        \"raised_exception\": false,\n",
       "        \"exception_message\": null,\n",
       "        \"exception_traceback\": null\n",
       "      },\n",
       "      \"success\": true\n",
       "    },\n",
       "    {\n",
       "      \"result\": {\n",
       "        \"element_count\": 15429,\n",
       "        \"unexpected_count\": 0,\n",
       "        \"unexpected_percent\": 0.0,\n",
       "        \"partial_unexpected_list\": []\n",
       "      },\n",
       "      \"meta\": {},\n",
       "      \"expectation_config\": {\n",
       "        \"meta\": {},\n",
       "        \"expectation_type\": \"expect_column_values_to_not_be_null\",\n",
       "        \"kwargs\": {\n",
       "          \"column\": \"Salary\"\n",
       "        }\n",
       "      },\n",
       "      \"exception_info\": {\n",
       "        \"raised_exception\": false,\n",
       "        \"exception_message\": null,\n",
       "        \"exception_traceback\": null\n",
       "      },\n",
       "      \"success\": true\n",
       "    },\n",
       "    {\n",
       "      \"result\": {\n",
       "        \"observed_value\": \"int64\"\n",
       "      },\n",
       "      \"meta\": {},\n",
       "      \"expectation_config\": {\n",
       "        \"meta\": {},\n",
       "        \"expectation_type\": \"expect_column_values_to_be_of_type\",\n",
       "        \"kwargs\": {\n",
       "          \"column\": \"Salary\",\n",
       "          \"type_\": \"int\"\n",
       "        }\n",
       "      },\n",
       "      \"exception_info\": {\n",
       "        \"raised_exception\": false,\n",
       "        \"exception_message\": null,\n",
       "        \"exception_traceback\": null\n",
       "      },\n",
       "      \"success\": true\n",
       "    },\n",
       "    {\n",
       "      \"result\": {\n",
       "        \"element_count\": 15429,\n",
       "        \"missing_count\": 0,\n",
       "        \"missing_percent\": 0.0,\n",
       "        \"unexpected_count\": 0,\n",
       "        \"unexpected_percent\": 0.0,\n",
       "        \"unexpected_percent_nonmissing\": 0.0,\n",
       "        \"partial_unexpected_list\": []\n",
       "      },\n",
       "      \"meta\": {},\n",
       "      \"expectation_config\": {\n",
       "        \"meta\": {},\n",
       "        \"expectation_type\": \"expect_column_values_to_match_regex\",\n",
       "        \"kwargs\": {\n",
       "          \"column\": \"Country\",\n",
       "          \"regex\": \"America|India|Another|China\"\n",
       "        }\n",
       "      },\n",
       "      \"exception_info\": {\n",
       "        \"raised_exception\": false,\n",
       "        \"exception_message\": null,\n",
       "        \"exception_traceback\": null\n",
       "      },\n",
       "      \"success\": true\n",
       "    }\n",
       "  ],\n",
       "  \"success\": true\n",
       "}"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kag_ge.to_csv('kag.csv')\n",
    "import json\n",
    "ge.validate(ge.read_csv('kag.csv'), \n",
    "    expectation_suite=json.load(\n",
    "        open('kaggle_expectations.json')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using pytest with pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How it works..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There's more..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Tests with Hypothesis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to do it..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "### How it works..."
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "comment_questions,-all",
   "main_language": "python",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
